#!/usr/bin/perl -w

#########################################################################
#									#
# Name:	 	threaded_lems.pl (Perl Program)				#
#									#
# Completed:	8th October, 2003.					#
#									#
# Updated:	25th January, 2005.					#
#									#
# Author:	Simon Edwards, Linuxha.net				#
#									#
# Arguments:	--application A --config F [--verbose][--file F]	#
#			[--check] [--detach] [--clustercfg F]		#
#									#
#		--check		Check that the configuration looks	#
#				valid or not and then exit with 0 or	#
# 				1 if there are problems. Messages	#
#				always go to stdout in this case.	#
#		--clustercfg	The name of the cluster configuration	#
#				file, defaulting to			#
#				/etc/cluster/clconf.xml.		#
#									#
# Purpose:	This daemon is run on a node that is running a 		#
#		particular package. Each package has a separate		#
#		daemon and it is used to handle events relating to 	#
#		that application.					#
#									#
# 		The events are all handled via modules loaded from	#
#		the configuration file - this is just a generalised	#
#		monitor of events and actions based on those events.	#
#									#
# Limitations:	1 - Single threaded - at some point migration to a	#
#		    later base version of perl should be used to 	#
#		    introduce threading support if possible.		#
#									#
# Environment:	None.							#
#									#
# Exit Codes:	0  - One or more files retrieved successfully.		#
#		>0 - A fatal error has occured.				#
# 									#
# Version:	@(#)1.0.0 Original (SE)>				#
# 		@(#)1.0.1 Initial changes for 0029 / 0030 (SE)>		#
# 		@(#)1.0.2 Change logging functionality /--check (SE)>	#
# 		@(#)1.0.3 Improved message handling (SE)>		#
# 		@(#)1.0.5 Support STOPAPP action (SE)>			#
# 		@(#)1.0.6 Fix bug 0045 (SE)>				#
# 		@(#)1.0.7 Minor code corrections (SE)>			#
# 		@(#)1.0.8 REMOVEMON action from monitors added (SE)>	#
#			  RUNCMD action supported for monitors		#
# 		@(#)1.0.9 Fixes for 0136 (SE)>				#
# 		@(#)1.1.0 --clustercfg and CLUSTER_CFG support (SE)>	#
# 		@(#)1.1.2 Improved port checking (SE)>			#
# 		@(#)1.1.3 Improved logdir checking and 0191 fix (SE)>	#
# 		@(#)1.1.4 Implement fix for 0207 (SE)>			#
#									#
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
#									#
# Complete SOCM Version History						#
#									#
# Version:	1.0.0							#
# Date:		7th October, 2003.					#
# Author:	Simon Edwards, Proprius Consulting Ltd.			#
# Change:	Original.						#
#									#
# Version:	1.0.1							#
# Date:		8th November, 2003.					#
# Author:	Simon Edwards, Proprius Consulting Ltd.			#
# Change:	Improvement 0030 proposes that the Lems system		#
#		provide an interface for getting status information	#
#		of all monitors, and detailed information per		#
#		monitor, if required.					#
#									#
#		Also the location of the logs makes use of the 		#
#		logs directory specified in the configuration file.	#
#		[logged as bug 0029]					#
# 									#
#		The GET_VSTAT implements 0030, and both this and	#
#		GET_STAT can take the name of a monitor to limit the	#
#		output just to this monitor.				#
#									#
# Version:	1.0.2							#
# Date:		21st November, 2003.					#
# Author:	Simon Edwards, Proprius Consulting Ltd.			#
# Change:	Implemented improved logging (does not log the 		#
#		times a NOP return code is generated!).			#
#									#
#		Also implemented --check to valid the configuratoin	#
#		file and exit with 0 or >0 (0 = ok, >0 = problem).	#
#									#
# Version:	1.0.3							#
# Date:		1st March, 2004.					#
# Author:	Simon Edwards, Proprius Consulting Ltd.			#
# Change:	Support pause action to take a break for the 		#
# 		monitor for X number of seconds...			#
# 		Also the PAUSE and RESUME commands accepted on the	#
# 		socket can take an optional module name to pause or	#
# 		resume just that module.				#
# 									#
#########################################################################

BEGIN {
	push @INC,"/usr/local/cluster/lib/perl";
}

use XML::Simple;
use Getopt::Long;
use File::Basename;
use lemsconfig;
use Data::Dumper;
use Time::HiRes qw(setitimer ITIMER_REAL);
use POSIX;
use cld_server;
use cld_client;
use clutils;
use Sys::Hostname;
use threads;
use threads::shared;
use Thread::Queue;

$Q=new Thread::Queue;
$Q2=new Thread::Queue;

my $PAUSED : shared;

$code_version=0.1;

$_PRG=basename($0);
$RCDATA="";
my $LOCKVAR : shared = 0;

sub checkotherlems {
my $port=$_[0];

	#################################################################
	# We scan other registered/valid applications, and if we	#
	# find any we see if a lems.local.xml file exists, and check 	#
	# the registered port - emmitting a warning if a clash exists.	#
	#################################################################

my ($dfd,$fd,$config3,$xml3,$cent,$err,$errstr);

	opendir($dfd,"/etc/cluster") or return -1;
	while($cent=readdir($dfd)) {
		next if $cent eq "." || $cent eq ".." || $cent eq $ARG_application;
		next if ! -d "/etc/cluster/$cent";
		next if ! -f "/etc/cluster/$cent/lems.local.xml";
		next if ! -f "/etc/cluster/$cent/appconf.xml";
		($xml3,$err,$errstr)=validate_application_cfg($cent);
		next if !defined($xml3);

		#########################################################
		# OK - xml3 contains a valid configuration, so attempt	#
		# to load in the lems configuration for this app...	#
		#########################################################
		$config3 = new lemsconfig("/etc/cluster/$cent/lems.local.xml");
		if(!defined($config3)) {
			logwarn("Ignoring application $cent - lems.local.xml is not valid.");
			next;
		}
		#########################################################
		# Get the port for the loaded configuration...		#
		#########################################################
		$port3=$config3->get_global("port");
		if(!defined($port3)) {
			logwarn("lems.local.xml for $cent does not define a port!");
			next;
		}
		if($port3 == $port) {
			errmsg("Lems port for application $cent also uses port $port!",56);
		}
		logmsg("No clash for application $cent - uses port $port3 not $port.");

	}
	closedir($dfd);
	return 0;
}

#########################################################################
# Will return 0 if the specified socket is not in use (no listener)	#
# or 1 if a tcp port on that socket is already open.			#
#########################################################################

sub checksocket {
my $port=$_[0];
my $host=$_[1];
my ($port2,$open,@o2,$o,@F);

	$port2=sprintf("%04X",$port);
	if($host eq hostname) {
		$local=1;
	} else {
		$local=0;
	}
	if($local) {
		my $fd;
		open($fd,"/proc/net/tcp");
		@o2=<$fd>;
		close($fd);
		$o=join("",@o2);
	} else {
		$o=`ssh $host cat /proc/net/tcp 2>/dev/null`;
	}

	#################################################################
	# Now $o has the output to scan for the port number in		#
	# question.							#
	#################################################################

	$open=0;
	for (split(/\n/m,$o)) {
		@F=split;
		next if $#F<3;
		next if $F[1] !~ /:/;
		if((split(/:/,$F[1]))[1] eq $port2) {
			if($F[3] eq "06") {
				logwarn("Found $port2 in TIME_WAIT - ignoring.");
			} else {
				$open=1;
			}
			last;
		}
	}
	return $open;
}

#########################################################################
# This will return the current UNIX time using the high resolution	#
# timer.								#
#########################################################################

sub getitime {
my $x;
	$x=Time::HiRes::time;
	return sprintf("%0.2f",$x);
}

#########################################################################
# This is the routine that is called to run a module or external	#
# program when required. It is passed three arguments:			#
# obj		The scheduler object we wish to run.			#
# ctime		The calculated "current" time				#
# check		The name of the check being run.			#
#########################################################################

sub handle_monitor {
my $obj=shift;
my $ctime=shift;
my $check=shift;

my ($r,$interval,$op,$mytime);
my (@f,$command,$incrthis);

	#################################################################
	# If the new function returned undef, then we issue a warning	#
	# once and do not schedule the check again.			#
	#################################################################

	if(!defined($obj->{OBJECT})) {
		logwarn("Check $check not initialised correctly - removed from monitoring");
		$obj->{EST}=0;
		return 0;
	}
	$r=$obj->{OBJECT}->check;
	$interval=$config->get_check_field($check,"INTERVAL");
	$op=$config->get_action($check,$r);
	$op =~ s/%RCDATA%/$RCDATA/g;

	if(!defined($op)) {
		$op="NOP";
	}
	if($op ne "NOP") {
		logmsg("Monitor $check returned: $r => [$op]");
	}

	#################################################################
	# Now we deal with the action list...				#
	#################################################################

	$incrthis=1;
	foreach $command (split(/;/,$op)) {
		@f=split(/\s+/,$command);
		$f[0]="\U$f[0]";
		next if $f[0] eq "NOP";
		if($f[0] eq "ABORT") {
			logmsg("Lems session aborted by action $check, RC=$r");
			exit 0;
		}
		if($f[0] eq "FAILOVER") {
			$conn=cld_client->new(PORT => $xml->{global}->{port}, INET => "127.0.0.1", 
				KEY => $server_key);
			if(!defined($conn)) {
				logwarn("Unable deliver FAILOVER message - no connection to cldaemon");
			} else {
				$resp=$conn->send_msg(MSG => "FAILOVER app=$ARG_application");
				if(defined($resp)) {
					logmsg("Cldaemon responded to FAILOVER with: $resp");
				} else {
					logwarn("No response from cldaemon for FAILOVER message");
				}
			}
			exit 0;
		}

		#########################################################
		# 0129 - Allow the administrator to run an external	#
		# 	 command as part of any action - most useful!	#
		# Note:  Such commands are run in the background, and	#
		#        any stdout/stderr are sent to the following	#
		#        files: /var/log/cluster/lems/app-mon.stdout	#
		#		/var/log/cluster/lems/app-mon.stderr	#
		#########################################################

		if($f[0] eq "RUNCMD") {
			my $s;

			$s=$command; $s =~ s/\s*RUNCMD\s+//;

			if(! -d "/var/log/cluster/lems") {
				mkdir "/var/log/cluster";
				mkdir "/var/log/cluster/lems";
			}
			$s.=">>/var/log/cluster/lems/$ARG_application-$mon.stdout ";
			$s.="2>>/var/log/cluster/lems/$ARG_application-$mon.stderr ";
			$s.="< /dev/null &";

			`$s`;
		}

		#########################################################
		# This allows a monitor to remove itself - i.e. it 	#
		# deems that it is no longer necessary.			#
		#########################################################

		if($f[0] eq "REMOVEMON") {
			delete $scheduler{$mon};
			return 0;
		}
		if($f[0] eq "STOPAPP" || $f[0] eq "HALTAPP") {
			$conn=cld_client->new(PORT => $xml->{global}->{port}, INET => "127.0.0.1", 
				KEY => $server_key);
			if(!defined($conn)) {
				logwarn("Unable deliver STOP_APP message - no connection to cldaemon");
			} else {
				$resp=$conn->send_msg(MSG => "STOP_APP app=$ARG_application");
				if(defined($resp)) {
					logmsg("Cldaemon responded to STOP_APP with: $resp");
				} else {
					logwarn("No response from cldaemon for STOP_APP message");
				}
			}
			exit 0;
		}
		if($f[0] eq "RUN" || $f[0] eq "START") {
			if(!exists($f[1])) {
				logwarn("Invalid RUN action syntax for check $check, RC=$r");
				next;
			}
			if(! $config->check_exists($f[1])) {
				logwarn("Specified RUN check $f[1] does not exist");
				logwarn("  [Action given for check $check, RC=$r]");
				next;
			}

			#################################################
			# We now set the EST entry for the specified	#
			# check, using $f[2] if set...			#
			#################################################

			my $interval=$config->get_check_field($f[1],"INTERVAL");
			if(exists($f[2])) {
				$mytime=$ctime+$f[2];
			} else {
				$mytime=$ctime+$interval;
			}
			$scheduler{$f[1]}->{EST}=$mytime;
			next;
		}

		#########################################################
		# Handle the PAUSE option - defaulting to a pause	#
		# of 30 seconds if no time is specified.		#
		#########################################################

		if($f[0] eq "PAUSE") {
			if(!exists($f[1])) {
				logwarn("Invalid RUN action syntax for check $check, RC=$r");
				next;
			}
			if(!exists($f[1])) {
				logwarn("Defaulting pause to 30 seconds.");
				$f[1]=30;
			}

			#################################################
			# We now set the EST entry for the specified	#
			# check, using $f[1] or the default of 30.	#
			#################################################

			$mytime=$ctime+$f[1];
			$obj->{EST}=$mytime;
			$incrthis=0;
			next;
		}

		#########################################################
		# The handling of STOP is quit similar to run...	#
		#########################################################

		if($f[0] eq "STOP" || $f[0] eq "HALT") {
			if(!exists($f[1])) {
				$scheduler{$check}->{EST}=0;
				logmsg("Check $check has been stopped, (RC=$r)");
				$incrthis=0;
			} else {
				if(! $config->check_exists($f[1])) {
					logwarn("Specified STOP check $f[1] does not exist");
					logwarn("  [Action given for check $check, RC=$r]");
					next;
				}
				$scheduler{$f[1]}->{EST}=0;
				logmsg("Check $f[1] has been stopped, (RC=$r)");
			}
			next;
		}
		logwarn("Unknown action $f[0] for check $check, RC=$r - ignored");

	}
	#################################################################
	# Finally set the next time we intend to run, if suitable.	#
	#################################################################
	$obj->{EST}=$ctime+$interval if $incrthis;

	return 0;
}

sub msg {
my $x=$ARG_verbose;

	$ARG_verbose=1;
	logmsg($_[0]);
	$ARG_verbose=$x;
}

sub usage {
	print "Usage: $_PRG --application X --config X [--file f] [--detach] \n";
	print "             [--check] [--verbose]\n";
	exit(1);
}

sub handle_cmd_msg {
my $monitor=shift;
my $msg=shift;
my ($mon,$r);

	foreach $mon (sort(keys(%scheduler))) {
		next if defined($monitor)  && $mon ne $monitor;
	
		eval '$r=$scheduler{$mon}->{OBJECT}->accept($msg);';
		if($@) {
			$cld_server->encode_msg(MSG => "NO_CMD_IMPLEMENTED: $@");
			return;
		}
		$r="NO_RETURN_VALUE" if !defined($r);
		$cld_server->encode_msg(MSG => $r);
		return;
	}
	$cld_server->encode_msg(MSG => "NO_SUCH_MONITOR: $monitor");
}

sub handle_get_stat {
my $monitor=shift;
my $txt="";
my ($mon,$r);
my ($interval,$module,$nextrun);

	foreach $mon (sort(keys(%scheduler))) {
		next if defined($monitor)  && $mon ne $monitor;
		my $interval=$config->get_check_field($mon,"INTERVAL");
		my $module=$config->get_check_field($mon,"MODULE");
		my $nextrun=$scheduler{$mon}->{EST};
		$txt.="$mon,$nextrun,$interval,$module\n";
	}
	chomp($txt);
	return $txt;
}
sub handle_verbosity_change {
my $mode=shift;

	$mode=uc($mode);
	if($mode ne "ON" && $mode ne "OFF") {
		$cld_server->encode_msg(MSG => "INVALID_ARG");
		return;
	}
	if($mode eq "ON") {
		$ARG_verbose=1;
	} else {
		$ARG_verbose=0;
	}
	$cld_server->encode_msg(MSG => "OK");
}

sub handle_log_cycle {
my $maxlogs=shift;
my ($n1,$n2);
my @exts=qw(Z gz bz2);

	if($maxlogs<1 || $maxlogs>99) {
		$cld_server->encode_msg(MSG => "INVALID_ARG");
		return;
	}
	if(! -f $ARG_logfile) {
		$cld_server->encode_msg(MSG => "LOGFILE_NOT_FILE");
		return;
	}

	#################################################################
	# Firstly we close stdout/stderr before attempting cycles	#
	#################################################################

	if(! -d dirname($ARG_logfile)) {
		print STDERR "Error: Lems unable to open $ARG_logfile - missing directory!\n";
		exit 1;
	}

	close(STDOUT);
	close(STDERR);

	$maxlogs--;
	while($maxlogs) {
		$n1=sprintf("$ARG_logfile.%03d",$maxlogs);
		$n2=sprintf("$ARG_logfile.%03d",$maxlogs-1);
		if(-f $n1) {
			unlink $n1;
		}
		if(-f $n2) {
			rename $n2,$n1;
		}
		for (@exts) {
			$n1=sprintf("$ARG_logfile.%03d.$_",$maxlogs);
			$n2=sprintf("$ARG_logfile.%03d.$_",$maxlogs-1);
			if(-f $n1) {
				unlink $n1;
			}
			if(-f $n2) {
				rename $n2,$n1;
			}
		}
		$maxlogs--;
	}
	$n2=sprintf("$ARG_logfile.%03d.$_",0);
	if(-f $ARG_logfile) {
		rename $ARG_logfile,$n2;
	}
	open(STDOUT,">>$ARG_logfile");
	open(STDERR,">>$ARG_logfile");
	$n1=$ARG_verbose;
	$ARG_verbose=1;
	logmsg("Log file cycled on user request.");
	$ARG_verbose=$n1;
	$cld_server->encode_msg(MSG => "OK");
}

sub handle_remove_monitor {
my $monitor=shift;
my ($mon,$r);

	return if !defined($monitor);
	#################################################################
	# We scan through the list of elements in the scheduler,	#
	# and remove them...						#
	#################################################################
	
	foreach $mon (sort(keys(%scheduler))) {
		next if $mon ne $monitor;

		delete $scheduler{$mon};
		$r=$cld_server->encode_msg(MSG => "$mon REMOVED");
		logmsg("Removed monitor \"$mon\" by user request.");
		return;
	}
	$r=$cld_server->encode_msg(MSG => "NO_SUCH_MONITOR: $monitor");
}


sub handle_pause_monitor {
my $monitor=shift;

	return if !defined($monitor);
	foreach $mon (sort(keys(%scheduler))) {
		next if $mon ne $monitor;

		#########################################################
		# Ok we are now at the monitor, so set the 		#
		# interval to 0.					#
		#########################################################
		$scheduler{$mon}->{EST}=0;
		$r=$cld_server->encode_msg(MSG => "$mon PAUSED");
		logmsg("Paused monitor \"$mon\" by user request.");
		return;
	}
	$r=$cld_server->encode_msg(MSG => "NO_SUCH_MONITOR: $monitor");
}

sub handle_resume_monitor {
my $monitor=shift;

	return if !defined($monitor);
	foreach $mon (sort(keys(%scheduler))) {
		next if $mon ne $monitor;

		#########################################################
		# Ok we are now at the monitor, so set the 		#
		# interval to 0.					#
		#########################################################
		my $interval=$config->get_check_field($mon,"INTERVAL");
		$scheduler{$mon}->{EST}=time+$interval;
		$r=$cld_server->encode_msg(MSG => "$mon RESUMED (in $interval secs)");
		logmsg("Resumed monitor \"$mon\" by user request.");
		return;
	}
	$r=$cld_server->encode_msg(MSG => "NO_SUCH_MONITOR: $monitor");
}

#########################################################################
# We are allowed to 'add' a monitor assuming the following is true -	#
# 1] The monitor exists in the configuration file (which is rescanned).	#
# 2] The monitor is not already in use.					#
#########################################################################

sub handle_add_monitor {
my $monitor=shift;
my ($mon,$r,$config2,@f);
my ($programs2,$modules2);
my (@check_list2,$check2,$found);

	return if !defined($monitor);
	logmsg("*** INSTALL MODULE $monitor START ***");
	foreach $mon (sort(keys(%scheduler))) {
		if($mon eq $monitor) {
			$r=$cld_server->encode_msg(MSG => "ALREADY_EXISTS $monitor");
			logwarn("Attempted to add monitor that already exists.");
			logmsg("*** INSTALL MODULE $monitor END ***");
			return;
		}
	}

	#################################################################
	# If we get here then the monitor does not currently exist,	#
	# so we check for details in the configuration.			#
	#################################################################

	$config2 = new lemsconfig($ARG_config);
	if(!defined($config2)) {
		logwarn("Invalid Configuration file load attempt!");
		logmsg("*** INSTALL MODULE $monitor END ***");
		$r=$cld_server->encode_msg(MSG => "INVALID CFGFILE");
		return;
	}

	#################################################################
	# Ok the configuration file is valid, so we now attempt to	#
	# get the monitor details from it for this monitor.		#
	#################################################################
	
	$modules2=$config2->get_global("modules");
	$programs2=$config2->get_global("programs");

	if(!defined($modules2)) {
		logwarn("Global variable \"modules\" not set in new config!");
		logmsg("*** INSTALL MODULE $monitor END ***");
		$r=$cld_server->encode_msg(MSG => "INVALID CFGFILE");
		return;
	}
	if(!defined($programs2)) {
		logwarn("Global variable \"programs\" not set in new config!");
		logmsg("*** INSTALL MODULE $monitor END ***");
		$r=$cld_server->encode_msg(MSG => "INVALID CFGFILE");
		return;
	}

	logmsg("New configuration using modules from  : $modules2");
	logmsg("New configuration using programs from : $programs2");

	#################################################################
	# We need to check details of our monitor before we accept	#
	# these new values...						#
	#################################################################

	@check_list2=$config2->get_check_list;

	$found=0;
	foreach $check2 (@check_list2) {
		if($check2 eq $monitor) {
			$found=1; last;
		}
	}
	if(!$found) {
		logwarn("New configuration does not contain monitor \"$monitor\".");
		logwarn("Aborting configuration changes.");
		logmsg("*** INSTALL MODULE $monitor END ***");
		$r=$cld_server->encode_msg(MSG => "INVALID CFGFILE");
		return;
	}

	#################################################################
	# We now get the details for the module we wish to add.		#
	#################################################################

	$type2=$config2->get_check_field($monitor,"TYPE");
	$module2=$config2->get_check_field($monitor,"MODULE");
	$interval2=$config2->get_check_field($monitor,"INTERVAL");
	if(!defined($type2)) {
		logwarn("Missing element \"type\" for check $monitor.");
		logmsg("*** INSTALL MODULE $monitor END ***");
		$r=$cld_server->encode_msg(MSG => "INVALID CFGFILE");
		return;
	}
	if(!defined($module2)) {
		logwarn("Missing element \"module\" for check $monitor.");
		logmsg("*** INSTALL MODULE $monitor END ***");
		$r=$cld_server->encode_msg(MSG => "INVALID CFGFILE");
		return;
	}
	if(!defined($interval2)) {
		logwarn("Missing element \"interval\" for check $monitor.");
		logmsg("*** INSTALL MODULE $monitor END ***");
		$r=$cld_server->encode_msg(MSG => "INVALID CFGFILE");
		return;
	}

	#########################################################################
	# Now we check to validate that "type" is "internal" or "program",	#
	# and error if not.							#
	#########################################################################

	if($type2 ne "internal" && $type2 ne "program") {
		logwarn("The \"type\" element must be \"internal\" or \"program\".");
		logmsg("*** INSTALL MODULE $monitor END ***");
		$r=$cld_server->encode_msg(MSG => "INVALID CFGFILE");
		return;
	}

	#########################################################################
	# Check that the specified internal or program actually exists...	#
	#########################################################################

	if($type2 eq "internal") {
		@f=split(/\s+/,$module2);
		if(! -f "$modules2/$f[0].pm") {
			logwarn("Missing module $modules2/$f[0].pm");
			logmsg("*** INSTALL MODULE $monitor END ***");
			$r=$cld_server->encode_msg(MSG => "INVALID CFGFILE");
			return;
		}
	} else {
		@f=split(/\s+/,$module2);
		if(! -f "$programs2/$f[0]") {
			logwarn("Missing program $programs2/$f[0]");
			logmsg("*** INSTALL MODULE $monitor END ***");
			$r=$cld_server->encode_msg(MSG => "INVALID CFGFILE");
			return;
		}
	}
	logmsg("Validated new monitor entry $monitor successfully.");

	if($type2 eq "internal") {
		my $mod=$config2->get_check_field($monitor,"MODULE");
		@f=split(/\s+/,$mod);
		{
			no strict;
			eval 'require "$modules2/$f[0].pm";';
			if($@) {
				errmsg("Unable to load module $f[0].pm\n   Error: $@",20);
			}
			$loaded_modules{$f[0]}=$monitor;
			logmsg("Successfully loaded module $f[0].pm");
		}
	}

	$scheduler{$monitor}={};
	if($interval2<0.5) {
		$scheduler{$monitor}->{EST}=0;
	} else {
		$scheduler{$monitor}->{EST}=time+$interval2;
	}

	#################################################################
	# Finally create the new object to represent this check.	#
	#################################################################

	if($type2 eq "internal") {
		my ($obj,$myobj);
		@f=split(/\s+/,$module2);

		#########################################################
		# Since the args are strings, we quote them before	#
		# doing anything else...				#
		#########################################################
		for (@f[1..$#f]) {
			$_="\"$_\"";
		}

		$obj="new $f[0]"."(".join(",",@f[1..$#f]).");";
		{
			no strict;
			$myobj=eval $obj;

		}
		if($@) {
			errmsg("Unable to create monitor object.\n     Error: $@",22);
			logmsg("*** INSTALL MODULE $monitor END ***");
			$cld_server->encode_msg(MSG => "WARNING");
			return;
		}
		#########################################################
		# If the module did not initialise, then we raise a 	#
		# warning here.						#
		#########################################################
		if(!defined($myobj)) {
			logwarn("Monitor $monitor (object type=$f[0]) not initialised.");
			logmsg("Success monitor add attempt - but no object returned.");
			logmsg("*** INSTALL MODULE $monitor END ***");
			$cld_server->encode_msg(MSG => "WARNING");
			return;
		}

		$scheduler{$monitor}->{OBJECT}=$myobj;
		if($interval>=0.5) {
			logmsg("Scheduled $monitor for ".$scheduler{$monitor}->{EST}." (object type=$f[0])");
		} else {
			logmsg("Created $monitor - not scheduled");
		}
	} else {
		$scheduler{$monitor}->{OBJECT}=undef;
		if($interval>=0.5) {
			logmsg("Scheduled $monitor for ".$scheduler{$monitor}->{EST}." (external program $f[0])");
		} else {
			logmsg("Created $monitor - not scheduled");
		}
	}

	#################################################################
	# Finally change the renaming globals and issue messages.	#
	#################################################################
	$MODDIR=$modules2;
	$PROGDIR=$programs2;

	logmsg("Successfully added new monitor $monitor.");
	logmsg("*** INSTALL MODULE $monitor END ***");
	$cld_server->encode_msg(MSG => "OK");
}

#########################################################################
# The vstat routine is provided to meet improvement 0030 - using it	#
# will return detailed per-monitor information, if that monitor		#
# supports a 'stat' method.						#
#########################################################################

sub handle_get_vstat {
my $monitor=shift;
my $txt="";
my ($mon,$r,$cmd,$eval_res);
my ($interval,$module,$nextrun);

	foreach $mon (sort(keys(%scheduler))) {
		next if defined($monitor)  && $mon ne $monitor;
		my $interval=$config->get_check_field($mon,"INTERVAL");
		my $module=$config->get_check_field($mon,"MODULE");
		if(!$PAUSED) {
			$nextrun=$scheduler{$mon}->{EST};
		} else {
			$nextrun=0;
		}
		$txt.="$mon,$nextrun,$interval,$module\n";
		$cmd='$scheduler{$mon}->{OBJECT}->stat';
		$eval_res=eval $cmd;
		chomp($eval_res) if defined($eval_res);
		if($@) {
			$txt.="<$mon-detail-start>\n<$mon-detail-end>\n";
		} else {
			$txt.="<$mon-detail-start>\n$eval_res\n<$mon-detail-end>\n";
		}
	}
	chomp($txt);
	return $txt;
}

$ARG_config = undef;
$ARG_application = undef;
$ARG_verbose = 0;
$ARG_detach=0;
$ARG_check=0;
$ARG_logfile = undef;
$ARG_clustercfg = undef;

if(!GetOptions(
	"application=s"	=> \$ARG_application,
	"verbose"	=> \$ARG_verbose,
	"file=s"	=> \$ARG_logfile,
	"detach"	=> \$ARG_detach,
	"check"		=> \$ARG_check,
	"clustercfg=s"	=> \$ARG_clustercfg,
	"config=s"	=> \$ARG_config)) {
	usage;
}

if(!defined($ARG_config) || !defined($ARG_application)) {
	usage;
}

$ARG_clustercfg="/etc/cluster/clconf.xml" if !defined($ARG_clustercfg);
$ENV{CLUSTER_CFG}=$ARG_clustercfg;

if(! -f $ARG_config) {
	print STDERR "Error: Configuration file does not exist!\n";
	exit(2);
}

#########################################################################
# If we've specified detach then we need to fork/setid now, and also	#
# ensure the process has opened STDOUT/STDERR as appropriate.		#
#########################################################################

$ARG_detach=0 if $ARG_check;
if($ARG_detach) {
	if($pid=fork) {
		exit;
	} else {
		die "cannot fork: $!" unless defined $pid;
		POSIX::setsid();
	}
}

$0="lems-$ARG_application";

#########################################################################
# We now read in the contents of the configuration file, which		#
# has the following format:						#
#									#
# <?xml version="1.0">							#
# <lems_config>								#
#	<globals modules="/home/sedwards/tmp/lems/modules"		#
#		 programs="/home/sedwards/tmp/lems/programs"		#
#		 logs="/home/sedwards/tmp/lems/logs"			#
#		 port="8900"						#
#	/>								#
#									#
#	<check>								#
#		<name>ip</name>						#
#		<type>internal</type>					#
#		<module>ip_module args</module>				#
#		<interval>2</interval>					#
#		<action_list>						#
#			<action rc="0" action="NOP"/>			#
#			<action rc="1" action="RUN move_ip"/>		#
#			<action rc="2" action="STOP"/>			#	
#		</action_list>						#
#	</check>							#
# </lems_config>							#
#									#
# Each <check> can have an interval between 0.5 - 3600+ seconds - 	#
# though unless it is an "internal" check, a interval of less than 10	#
# is not recommended.							#
#									#
# All internal checks are simple calls to the object function - no	#
# threading will be used. This means that each action must complete	#
# as quickly as possible.						#
#									#
# The lems program will always sleep at most 1/2 of the shortest	#
# internval - which would be 1 second in the above case.		#
# It will then increment all the timers associated with the actions, 	#
# and then call them in sequence given in the configuration file.	#
#									#
# The actions are based on the return code, and multiple actions are	#
# possible using ; between them. The actions that are supported are:	#
#									#
# NOP		- No Operation (do nothing - default)			#
# RUN label args- Execute specified remote application with args.	#
# STOP [label]	- Halt subsequent calls to current or labeled check.	#
# START label   - Start calls again to specified label.			#
# ABORT         - Exit lems - this will cause all monitoring to stop.	#
#									#
# A typical configuration for a standard IP setup, would look something	#
# like the following:							#
#									#
# link_check IP	- Checks that the link level connection currently	#
#		  used for the specified IP address is up. Actions:	#
#		  0 - NOP (everything is OK)				#
#		  1 - RUN migrate_ip IP - link is down, attempt to move	#
#		      to other card specified for this network.		#
#		  2 - NOP (network down, can do nothing, remain calm!)	#
# link_check_flag IP - Checks to see if the link_check ignore flag has	#
#		       been removed, and if so, starts the link		#
#		       checking again.					#
# 		       0 - NOP (no change to configuration)		#
#		       1 - STOP link_check Stop monitoring since the	#
#			   ignore flag for the network has been found.	#
#		       2 - START link_check Start monitoring since the	#
#		     	   ignore flag has been removed.		#
#									#
# Notes:								#
# [1]	The internal modules can loaded into the lems environment, and	#
#	hence can maintain state, since they are instances of objects.	#
#	This makes things like the above STOP/START actions much 	#
#	easier to implement.						#
#									#
#		        						#
#########################################################################

$config = new lemsconfig($ARG_config);
errmsg("Invalid Configuration file specified",2) if ! defined $config;

$modules=$config->get_global("modules");
$programs=$config->get_global("programs");
$logs=$config->get_global("logs");
$port=$config->get_global("port");

#########################################################################
# Now we have information on the losgs, we use it if possible, and if	#
# not will default to logging information to "/var/log/cluster/lems".	#
# However if we are running in --check mode then we do nothing and	#
# ensure the log file points to STDOUT.					#
#########################################################################

($xml,$err_code,$err_msg)=validate_cluster_cfg($ARG_clustercfg);
if(!defined($xml)) {
	errmsg("$err_msg",$err_code);
}

#########################################################################
# Check the bare minimum details exist in the configuration file.	#
#########################################################################

if(!exists($xml->{global}->{version}) ||
	!exists($xml->{global}->{data}) ||
	!exists($xml->{global}->{datadetail}) ||
	!exists($xml->{global}->{key}) ||
	!exists($xml->{global}->{port}) ||
	!exists($xml->{global}->{logdir})) {
	errmsg("Missing attribute from global section of clconf.xml",4);
}

if($xml->{global}->{version}>$code_version) {
	errmsg("Configuration file for later version of software!",5);
}

$myhostname=hostname;
@nodes=keys(%{$xml->{node}});
if($nodes[0] eq $myhostname) {
	$lnode=$myhostname;
	$rnode=$nodes[1];
} else {
	$lnode=$myhostname;
	$rnode=$nodes[0];
}

$server_key=$xml->{global}->{key};
if(!$ARG_check) {
	if(defined($logs)) {
		$LOGDIR=$logs
	} else {
		$LOGDIR="/var/log/cluster/lems";
	}

	# close(STDOUT); close(STDERR);
	if(!defined($ARG_logfile)) {
		if(! -d $LOGDIR) {
			print STDERR "Error: Missing directory in path $LOGDIR!\n";
			print STDERR "Will use /tmp instead!\n";
			$LOGDIR="/tmp";
		}
		open(STDOUT,">>$LOGDIR/lems-$ARG_application.log");
		open(STDERR,">>$LOGDIR/lems-$ARG_application.log");
		$ARG_logfile="$LOGDIR/lems-$ARG_application.log";
	} else {
		if(! -d dirname($ARG_logfile)) {
			print STDERR "Error: Missing directory in path ".dirname($ARG_logfile)."!\n";
			print STDERR "Will use /tmp instead!\n";
			$ARG_logfile="/tmp/".basename($ARG_logfile);
		}
		open(STDOUT,">>$ARG_logfile");
		open(STDERR,">>$ARG_logfile");
	}
}

if(!defined($modules)) {
	errmsg("Global variable \"modules\" not set.",5);
}
if(!defined($programs)) {
	errmsg("Global variable \"programs\" not set.",5);
}
if(!defined($logs)) {
	errmsg("Global variable \"logs\" not set.",5);
}

if(!defined($port)) {
	errmsg("Global variable \"port\" not set.",5);
}
logmsg("Using modules from  : $modules");
logmsg("Using programs from : $programs");
logmsg("Writing logs to     : $logs");

logmsg("Validating port not open locally...");
$popen=checksocket($port,$lnode);
if($popen) {
	errmsg("Port $port open on local host!",67);
}
#########################################################################
# 0191: Comment out the remote check since it makes lems hang for up 	#
#	to 30 seconds if remote node is not available.			#
#########################################################################
if($ARG_check) {
	logmsg("Validating port not open on $rnode...");
	$popen=checksocket($port,$rnode);
	if($popen) {
		logwarn("Port $port open on $rnode.");
	}
}

checkotherlems($port);

logmsg("Listening on port   : $port");

#########################################################################
# Now we check that the directories for the globals exist, and if not	#
# we actually attempt to create them.					#
#########################################################################

if(! -f "/sbin/cluster/utils/mkdir") {
	errmsg("Unable to find /sbin/cluster/utils/mkdir utility!",6);
}

if(! -d $modules) {
	$r=`/sbin/cluster/utils/mkdir root bin 755 $modules 2>&1`;
	if($?>>8) {
		errmsg("Unable to create directory $modules.",7);
	} else {
		logwarn("Directory $modules created.");
	}
}
$MODDIR=$modules;

if(! -d $programs) {
	$r=`/sbin/cluster/utils/mkdir root bin 755 $programs 2>&1`;
	if($?>>8) {
		errmsg("Unable to create directory $programs.",7);
	} else {
		logwarn("Directory $programs created.");
	}
}
$PROGDIR=$programs;

if(! -d $logs) {
	$r=`/sbin/cluster/utils/mkdir root bin 755 $logs 2>&1`;
	if($?>>8) {
		errmsg("Unable to create directory $logs.",7);
	} else {
		logwarn("Directory $logs created.");
	}
}
logmsg("Global initialisation complete.");
$LOGDIR=$logs;

#########################################################################
# Now the next step is to check the details of each of the monitors,	#
# and ensure the specified commands exist, and that the intervals	#
# are valid.								#
#########################################################################

@check_list=$config->get_check_list;

foreach $check (@check_list) {
	logmsg("Validating monitor entry $check...");

	#################################################################
	# Get the fields and check they are defined for current entry.	#
	#################################################################

	$type=$config->get_check_field($check,"TYPE");
	$module=$config->get_check_field($check,"MODULE");
	$interval=$config->get_check_field($check,"INTERVAL");
	if(!defined($type)) {
		errmsg("Missing element \"type\" for check $check.",8);
	}
	if(!defined($module)) {
		errmsg("Missing element \"module\" for check $check.",8);
	}
	if(!defined($interval)) {
		errmsg("Missing element \"interval\" for check $check.",8);
	}

	#########################################################################
	# Now we check to validate that "type" is "internal" or "program",	#
	# and error if not.							#
	#########################################################################

	if($type ne "internal" && $type ne "program") {
		errmsg("The \"type\" element must be \"internal\" or \"program\".",9);
	}

	#########################################################################
	# Check that the specified internal or program actually exists...	#
	#########################################################################

	if($type eq "internal") {
		@f=split(/\s+/,$module);
		if(! -f "$MODDIR/$f[0].pm") {
			errmsg("Missing module $modules/$f[0].pm",10);
		}
	} else {
		@f=split(/\s+/,$module);
		if(! -f "$PROGDIR/$f[0]") {
			errmsg("Missing program $programs/$f[0]",10);
		}
	}
	logmsg("Validated monitor entry $check successfully.");
}

#########################################################################
# If running in check mode we attempt to send the file to the remote	#
# node to ensure it also has a valid copy of the configuration.		#
#########################################################################

if($ARG_check) {
	logmsg("Check mode - transferring validated config to remote node.");
	`scp $ARG_config $rnode:$ARG_config >/dev/null 2>/dev/null`;
	if($?>>8) {
		logwarn("Unable to transfer configuration to remote node.");
	} else {
		logmsg("Configuration transferred successfully.");
	}
}

#########################################################################
# Now we have validated the contents of all monitors the next step is	#
# to ascertain the minimal interval to sleep for before waking up and 	#
# checking whether any process needs to be run.				#
# The interval used is 1/4 of the smallest interval, or 0.2 seconds -	#
# whichever is larger.							#
#########################################################################

$interval=60;
foreach $check (@check_list) {
	$cint=$config->get_check_field($check,"INTERVAL");
	if($cint>0.5 && $cint/4 < $interval) {
		$interval=$cint/4;
	}
}
$interval=0.2 if $interval<0.2;
$interval=sprintf("%0.1f",$interval);
logmsg("Calculated a check interval of $interval seconds.");
$SLEEP_FOR=$interval;

exit(0) if $ARG_check;
#########################################################################
# Now the interval has been defined we actually load all the modules	#
# and check we are able to make objects for each check (if applicable.)	#
#########################################################################

push @INC,$MODDIR;
%loaded_modules=();
foreach $check (@check_list) {
	$type=$config->get_check_field($check,"TYPE");
	if($type eq "internal") {
		$mod=$config->get_check_field($check,"MODULE");
		@f=split(/\s+/,$mod);
		if(exists($loaded_modules{$f[0]})) {
			logmsg("NOT loading $f[0] for check $check - already loaded for $loaded_modules{$f[0]}");
		} else {
			no strict;
			eval 'require "$MODDIR/$f[0].pm";';
			if($@) {
				errmsg("Unable to load module $f[0].pm\n   Error: $@",20);
			}
			$loaded_modules{$f[0]}=$check;
			logmsg("Successfully loaded module $f[0].pm");
		}
	}
}

#########################################################################
# Now the code for all the internal modules has been loaded the next	#
# step is to create the %scheduler hash. In this each element key is	#
# the name of a monitor, with the value being a reference to a hash	#
# containing the following fields:					#
# OBJECT	- The reference to the created object for this check.	#
# 		  [Only appears if the check is internal].		#
# EST		- Estimated time of next run, (including fractions of	#
#	 	  a second as well.					#
#########################################################################

%scheduler=();
$ctime=getitime;

foreach $check (@check_list) {

	$scheduler{$check}={};
	$interval=$config->get_check_field($check,"INTERVAL");

	#################################################################
	# Work out the next time to run, putting to 0 if we do not 	#
	# wish to run currently.					#
	#################################################################

	if($interval<0.5) {
		$scheduler{$check}->{EST}=0;
	} else {
		$scheduler{$check}->{EST}=$ctime+$interval;
	}
	$type=$config->get_check_field($check,"TYPE");
	$module=$config->get_check_field($check,"MODULE");

	#################################################################
	# For internal objects create some code to dynamically create	#
	# an instance of the object type, passing supplied parameters.	#
	#################################################################
	if($type eq "internal") {
		@f=split(/\s+/,$module);

		#########################################################
		# Since the args are strings, we quote them before	#
		# doing anything else...				#
		#########################################################
		for (@f[1..$#f]) {
			$_="\"$_\"";
		}

		$obj="new $f[0]"."(".join(",",@f[1..$#f]).");";
		{
			no strict;
			$myobj=eval $obj;

		}
		if($@) {
			errmsg("Unable to create monitor object.\n     Error: $@",22);
		}

		#########################################################
		# If the module did not initialise, then we raise a 	#
		# warning here.						#
		#########################################################
		if(!defined($myobj)) {
			logwarn("Monitor $check (object type=$f[0]) not initialised.");
			delete $scheduler{$check};
			next;
		}

		$scheduler{$check}->{OBJECT}=$myobj;
		if($interval>=0.5) {
			logmsg("Scheduled $check for ".$scheduler{$check}->{EST}." (object type=$f[0])");
		} else {
			logmsg("Created $check - not scheduled");
		}
	} else {
		$scheduler{$check}->{OBJECT}=undef;
		if($interval>=0.5) {
			logmsg("Scheduled $check for ".$scheduler{$check}->{EST}." (external program $f[0])");
		} else {
			logmsg("Created $check - not scheduled");
		}

	}
}

#########################################################################
# Rather than a main loop we now have one routine for reading the	#
# socket and another for handling the event monitoring.			#
# The socket handler will enqueue requests and await responses...	#
#########################################################################
$PAUSED = 0;

sub socket_handle_thread {
my ($fd,$msg,$valid_req,$cld_server);
	$cld_server=cld_server->new(PORT => $port, KEY => $server_key);
	if(!defined($cld_server)) {
		errmsg("Unable to create server on specified socket.",10);
	}
	# logmsg("Started local server on port $port");
	# $cld_server->non_block;

	local $SIG{ALRM}='IGNORE';	# Socket thread ignores SIGALARM
	while(1) {
		logmsg("Waiting for socket request.");
		$fd=$cld_server->accept;
		if(defined($fd)) {
			$msg=$cld_server->decode_msg;
			if(!defined($msg)) {
				logwarn("Invalid message format recieved");
				$cld_server->close_session;
			} else {
				# kill ALRM => $$;
				{ no warnings 'threads'; cond_signal($LOCKVAR); }
				#################################################
				# We've got a correctly encrypted message so	#
				# deal with the request...			#
				#################################################
				$valid_req=0;
				if($msg =~ /^GET_STAT/) {
					my @F=split(/\s+/,$msg);
					shift(@F);
					for (@F) {$_="\"$_\""; }
					$Q->enqueue("RR handle_get_stat(".join(",",@F).");");
					$cld_server->encode_msg(MSG => $Q2->dequeue);
					$cld_server->close_session;
					next;
				}
				if($msg =~ /^GET_VSTAT/) {
					my @F=split(/\s+/,$msg);
					shift(@F);
					for (@F) {$_="\"$_\""; }
					$Q->enqueue("RR handle_get_vstat(".join(",",@F).");");
					$cld_server->encode_msg(MSG => $Q2->dequeue);
					$cld_server->close_session;
					next;
				}
				if($msg =~ /^CMD\s*/) {
					my @F=split(/\s+/,$msg);
					$valid_req=1;
					if(!exists($F[2]) || $F[2] =~ /^\s+$/) {
						$cld_server->encode_msg(MSG => "MISSING_ARG");
					} else {
						handle_cmd_msg($F[1],join(" ",@F[2..$#F]));
					}
				}
				if($msg =~ /^REMOVE\s*/) {
					my @F=split(/\s+/,$msg);
					$valid_req=1;
					if(!exists($F[1]) || $F[1] =~ /^\s+$/) {
						$cld_server->encode_msg(MSG => "MISSING_ARG");
					} else {
						handle_remove_monitor($F[1]);
					}
				}
				if($msg =~ /^VERBOSE\s*/) {
					my @F=split(/\s+/,$msg);
					$valid_req=1;
					if(!exists($F[1]) || $F[1] =~ /^\s+$/) {
						$cld_server->encode_msg(MSG => "MISSING_ARG");
					} else {
						handle_verbosity_change($F[1]);
					}
				}
				if($msg =~ /^LOGCYCLE\s*/) {
					my @F=split(/\s+/,$msg);
					$valid_req=1;
					if(!exists($F[1]) || $F[1] =~ /^\s+$/) {
						$cld_server->encode_msg(MSG => "MISSING_ARG");
					} else {
						handle_log_cycle($F[1]);
					}
				}
				if($msg =~ /^INSTALL\s*/) {
					my @F=split(/\s+/,$msg);
					$valid_req=1;
					if(!exists($F[1]) || $F[1] =~ /^\s+$/) {
						$cld_server->encode_msg(MSG => "MISSING_ARG");
					} else {
						handle_add_monitor($F[1]);
					}
				}
				if($msg =~ /^PAUSE\s*/) {
					my @F=split(/\s+/,$msg);
					if(!exists($F[1]) || $F[1] =~ /^\s+$/) {
						$PAUSED=1;
						logmsg("ALL MONITORING PAUSED");
					} else {
						$Q->enqueue("handle_pause_monitor(\"$F[1]\");");
					}
					$valid_req=1;
				}
				if($msg =~ /^RESUME\s*/) {
					my @F=split(/\s+/,$msg);
					if(!exists($F[1]) || $F[1] =~ /^\s+$/) {
						$PAUSED=0;
						logmsg("ALL MONITORING RESUMED");
					} else {
						$Q->enqueue("handle_resume_monitor(\"$F[1]\");");
					}
					$valid_req=1;
				}
				if($msg eq "ABORT") {
					$PAUSED=0;
					$cld_server->encode_msg(MSG => "OK");
					$cld_server->close_session;
					logmsg("ALL MONITORING ABORTED");
					last;
				}
				$cld_server->encode_msg(MSG => "OK") if $valid_req;
				$cld_server->encode_msg(MSG => "Unknown request") if ! $valid_req;
				$cld_server->close_session;
			}
		}
	}
	#################################################################
	# We get here when the ABORT message has been gotton,		#
	# so we drop out now to remove the thread.			#
	#################################################################
	exit(0);

}

sub scheduler_thread {
my $int=0;
my $wait_time=0.1;  # First attempt just wait 0.1 of a seconds...
my $ctime;

	while(1) {
		#########################################################
		# Handle a queued event if available...			#
		#########################################################
		{
			lock($LOCKVAR);
			#  $SIG{ALRM}=sub { die "TIMEOUT" };
			# setitimer(ITIMER_REAL,$wait_time,0);
			# $xx=cond_timedwait($LOCKVAR,time+$wait_time);
			$xx=cond_timedwait($LOCKVAR,time+1);
		}
		$ev=$Q->dequeue_nb;
		if(defined($ev)) {
			logmsg("Handle queue event: $ev");
			@F=split(/\s+/,$ev);
			if($F[0] ne "RR") {
				eval "$ev";
			} else {
				logmsg("Want return value...");
				$ev=~ s/^RR //;
				$rr=eval "$ev";
				$Q2->enqueue($rr);
			}
		} else {
			logmsg("Timeout event...");
		}
		if(!$PAUSED) {
			$ctime=getitime;
			#########################################################
			# Handle any pending events...				#
			#########################################################
			foreach $mon (sort(keys(%scheduler))) {
				if($scheduler{$mon}->{EST} > 0 &&
					$scheduler{$mon}->{EST} < $ctime) {
					$r=handle_monitor($scheduler{$mon},$ctime,$mon);
					last if $r;
				}
			}
		}
		$ctime=getitime;
		#########################################################
		# Get the next time to run a module...			#
		#########################################################
		$smallest=undef;
		foreach $mon (sort(keys(%scheduler))) {
			if($scheduler{$mon}->{EST} > 0) {
				$smallest=$scheduler{$mon}->{EST} if !defined($smallest);
				$smallest=$scheduler{$mon}->{EST} if $smallest > $scheduler{$mon}->{EST};
			}
		}
		$time_wait=int($smallest-$ctime);
		$time_wait=1 if $time_wait < 1;
		logmsg("Waiting for $time_wait seconds...");
	}
	exit(0);
}

$SIG{ALRM}='IGNORE';	# Top level thread ignores SIGALARM
# Here we create the threads and let them run... if either abort then	#
# the program exits...							#

$sched_thread=threads->new(\&scheduler_thread);
$socket_thread=threads->new(\&socket_handle_thread);

$sched_thread->join;
$socket_thread->join;
logmsg("Lems Monitor Aborted!");
exit(0);
